/*     */ package com.zimbra.cs.store.file;
/*     */ 
/*     */ import com.zimbra.common.localconfig.DebugConfig;
/*     */ import com.zimbra.common.service.ServiceException;
/*     */ import com.zimbra.common.util.Log;
/*     */ import com.zimbra.common.util.Pair;
/*     */ import com.zimbra.common.util.ZimbraLog;
/*     */ import com.zimbra.cs.db.DbMailItem;
/*     */ import com.zimbra.cs.db.DbMailbox;
/*     */ import com.zimbra.cs.db.DbPool;
/*     */ import com.zimbra.cs.db.DbPool.DbConnection;
/*     */ import com.zimbra.cs.db.DbVolumeBlobs;
/*     */ import com.zimbra.cs.mailbox.MailServiceException;
/*     */ import com.zimbra.cs.store.MailboxBlob.MailboxBlobInfo;
/*     */ import com.zimbra.cs.util.SpoolingCache;
/*     */ import com.zimbra.cs.volume.Volume;
/*     */ import com.zimbra.cs.volume.Volume.Builder;
/*     */ import com.zimbra.cs.volume.Volume.VolumeMetadata;
/*     */ import com.zimbra.cs.volume.VolumeManager;
/*     */ import com.zimbra.znative.IO;
/*     */ import com.zimbra.znative.IO.FileInfo;
/*     */ import java.io.File;
/*     */ import java.io.IOException;
/*     */ import java.util.ArrayList;
/*     */ import java.util.Collection;
/*     */ import java.util.Collections;
/*     */ import java.util.Iterator;
/*     */ import java.util.LinkedHashMap;
/*     */ import java.util.List;
/*     */ import java.util.Map;
/*     */ import java.util.Map.Entry;
/*     */ import java.util.Set;
/*     */ import org.apache.commons.collections.MultiMap;
/*     */ import org.apache.commons.collections.map.MultiValueMap;
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ public class BlobDeduper
/*     */ {
/*  53 */   private boolean inProgress = false;
/*  54 */   private boolean stopProcessing = false;
/*  55 */   private int totalLinksCreated = 0;
/*  56 */   private long totalSizeSaved = 0L;
/*  57 */   private Map<Short, String> volumeBlobsProgress = new LinkedHashMap();
/*  58 */   private Map<Short, String> blobDigestsProgress = new LinkedHashMap();
/*     */   
/*  60 */   private static final BlobDeduper SINGLETON = new BlobDeduper();
/*     */   
/*     */ 
/*     */ 
/*     */   public static BlobDeduper getInstance()
/*     */   {
/*  66 */     return SINGLETON;
/*     */   }
/*     */   
/*     */   private Pair<Integer, Long> processDigest(String digest, Volume volume) throws ServiceException
/*     */   {
/*  71 */     DbPool.DbConnection conn = null;
/*     */     List<BlobReference> blobs;
/*     */     try {
/*  74 */       conn = DbPool.getConnection();
/*  75 */       blobs = DbVolumeBlobs.getBlobReferences(conn, digest, volume);
/*     */     } finally {
/*  77 */       DbPool.quietClose(conn);
/*     */     }
/*     */     
/*  80 */     if (blobs.size() > 1) {
/*  81 */       ZimbraLog.misc.debug("Deduping " + blobs.size() + " files for digest " + digest + " volume " + volume.getId());
/*  82 */       return deDupe(blobs); }
/*  83 */     if (blobs.size() == 1)
/*     */     {
/*  85 */       markBlobAsProcessed((BlobReference)blobs.get(0));
/*     */     }
/*  87 */     return new Pair(Integer.valueOf(0), Long.valueOf(0L));
/*     */   }
/*     */   
/*     */   private Pair<Integer, Long> deDupe(List<BlobReference> blobs) throws ServiceException {
/*  91 */     int linksCreated = 0;
/*  92 */     long sizeSaved = 0L;
/*  93 */     long srcInodeNum = 0L;
/*  94 */     String srcPath = null;
/*     */     
/*  96 */     for (BlobReference blob : blobs) {
/*  97 */       if (blob.isProcessed()) {
/*  98 */         String path = FileBlobStore.getBlobPath(blob.getMailboxId(), blob.getItemId(), blob.getRevision(), blob.getVolumeId());
/*     */         
/*     */         try
/*     */         {
/* 102 */           IO.FileInfo fileInfo = IO.fileInfo(path);
/* 103 */           if (fileInfo != null) {
/* 104 */             srcInodeNum = fileInfo.getInodeNum();
/* 105 */             srcPath = path;
/* 106 */             break;
/*     */           }
/*     */         }
/*     */         catch (IOException e) {}
/*     */       }
/*     */     }
/*     */     
/* 113 */     if (srcInodeNum == 0L)
/*     */     {
/*     */ 
/* 116 */       MultiMap inodeMap = new MultiValueMap();
/* 117 */       for (BlobReference blob : blobs) {
/* 118 */         String path = FileBlobStore.getBlobPath(blob.getMailboxId(), blob.getItemId(), blob.getRevision(), blob.getVolumeId());
/*     */         
/*     */         try
/*     */         {
/* 122 */           IO.FileInfo fileInfo = IO.fileInfo(path);
/* 123 */           if (fileInfo != null) {
/* 124 */             inodeMap.put(Long.valueOf(fileInfo.getInodeNum()), path);
/* 125 */             blob.setFileInfo(fileInfo);
/*     */           }
/*     */         }
/*     */         catch (IOException e) {}
/*     */       }
/*     */       
/*     */ 
/* 132 */       int maxPaths = 0;
/*     */       
/* 134 */       Iterator<Map.Entry<Long, Collection<String>>> iter = inodeMap.entrySet().iterator();
/* 135 */       while (iter.hasNext()) {
/* 136 */         Map.Entry<Long, Collection<String>> entry = (Map.Entry)iter.next();
/* 137 */         if (((Collection)entry.getValue()).size() > maxPaths) {
/* 138 */           maxPaths = ((Collection)entry.getValue()).size();
/* 139 */           srcInodeNum = ((Long)entry.getKey()).longValue();
/* 140 */           srcPath = (String)((Collection)entry.getValue()).iterator().next();
/*     */         }
/*     */       }
/*     */     }
/* 144 */     if (srcInodeNum == 0L) {
/* 145 */       return new Pair(Integer.valueOf(0), Long.valueOf(0L));
/*     */     }
/*     */     
/*     */ 
/* 149 */     String holdPath = srcPath + "_HOLD";
/* 150 */     File holdFile = new File(holdPath);
/*     */     try {
/* 152 */       IO.link(srcPath, holdPath);
/*     */       
/* 154 */       for (BlobReference blob : blobs) {
/* 155 */         if (!blob.isProcessed())
/*     */         {
/*     */ 
/* 158 */           String path = FileBlobStore.getBlobPath(blob.getMailboxId(), blob.getItemId(), blob.getRevision(), blob.getVolumeId());
/*     */           
/*     */           try
/*     */           {
/* 162 */             if (blob.getFileInfo() == null) {
/* 163 */               blob.setFileInfo(IO.fileInfo(path));
/*     */             }
/*     */           }
/*     */           catch (IOException e) {}
/*     */           
/* 168 */           if (blob.getFileInfo() != null)
/*     */           {
/*     */ 
/* 171 */             if (srcInodeNum == blob.getFileInfo().getInodeNum()) {
/* 172 */               markBlobAsProcessed(blob);
/*     */ 
/*     */             }
/*     */             else
/*     */             {
/*     */ 
/* 178 */               String tempPath = path + "_TEMP";
/* 179 */               File tempFile = new File(tempPath);
/*     */               try {
/* 181 */                 IO.link(holdPath, tempPath);
/* 182 */                 File destFile = new File(path);
/* 183 */                 tempFile.renameTo(destFile);
/* 184 */                 markBlobAsProcessed(blob);
/* 185 */                 linksCreated++;
/* 186 */                 sizeSaved += blob.getFileInfo().getSize();
/*     */               } catch (IOException e) {
/* 188 */                 ZimbraLog.misc.warn("Ignoring the error while deduping " + path, e);
/*     */               }
/*     */               finally {}
/*     */             }
/*     */           }
/*     */         }
/*     */       }
/*     */     } catch (IOException e) {
/* 196 */       ZimbraLog.misc.warn("Ignoring the error while creating a link for " + srcPath, e);
/*     */     } finally {
/* 198 */       if (holdFile.exists()) {
/* 199 */         holdFile.delete();
/*     */       }
/*     */     }
/* 202 */     return new Pair(Integer.valueOf(linksCreated), Long.valueOf(sizeSaved));
/*     */   }
/*     */   
/*     */   private void markBlobAsProcessed(BlobReference blob) throws ServiceException {
/* 206 */     DbPool.DbConnection conn = null;
/*     */     try {
/* 208 */       conn = DbPool.getConnection();
/* 209 */       DbVolumeBlobs.updateProcessed(conn, blob.getId(), true);
/* 210 */       conn.commit();
/*     */     } finally {
/* 212 */       DbPool.quietClose(conn);
/*     */     }
/*     */   }
/*     */   
/*     */   public synchronized void stopProcessing() {
/* 217 */     if (this.inProgress) {
/* 218 */       ZimbraLog.misc.info("Setting stopProcessing flag.");
/* 219 */       this.stopProcessing = true;
/*     */     }
/*     */   }
/*     */   
/*     */   private synchronized boolean isStopProcessing() {
/* 224 */     return this.stopProcessing;
/*     */   }
/*     */   
/*     */   public synchronized boolean isRunning() {
/* 228 */     return this.inProgress;
/*     */   }
/*     */   
/*     */   private synchronized void resetProgress() {
/* 232 */     this.inProgress = false;
/* 233 */     this.stopProcessing = false;
/*     */   }
/*     */   
/*     */   private synchronized void incrementCountAndSize(int count, long size) {
/* 237 */     this.totalLinksCreated += count;
/* 238 */     this.totalSizeSaved += size;
/*     */   }
/*     */   
/*     */   public synchronized Pair<Integer, Long> getCountAndSize() {
/* 242 */     return new Pair(Integer.valueOf(this.totalLinksCreated), Long.valueOf(this.totalSizeSaved));
/*     */   }
/*     */   
/*     */   public synchronized Map<Short, String> getVolumeBlobsProgress() {
/* 246 */     return this.volumeBlobsProgress;
/*     */   }
/*     */   
/*     */   public synchronized void setVolumeBlobsProgress(short volumeId, String str) {
/* 250 */     this.volumeBlobsProgress.put(Short.valueOf(volumeId), str);
/*     */   }
/*     */   
/*     */   public synchronized Map<Short, String> getBlobDigestsProgress() {
/* 254 */     return this.blobDigestsProgress;
/*     */   }
/*     */   
/*     */   public synchronized void setBlobDigestsProgress(short volumeId, String str) {
/* 258 */     this.blobDigestsProgress.put(Short.valueOf(volumeId), str);
/*     */   }
/*     */   
/*     */   public void resetVolumeBlobs(List<Short> volumeIds) throws ServiceException {
/* 262 */     synchronized (this) {
/* 263 */       if (this.inProgress) {
/* 264 */         throw MailServiceException.TRY_AGAIN("Dedupe is in progress. Stop the dedupe and then run reset again.");
/*     */       }
/* 266 */       this.inProgress = true;
/*     */     }
/* 268 */     DbPool.DbConnection conn = null;
/*     */     try {
/* 270 */       conn = DbPool.getConnection();
/* 271 */       Iterator i$; if (volumeIds.isEmpty())
/*     */       {
/* 273 */         DbVolumeBlobs.deleteAllBlobRef(conn);
/*     */         
/* 275 */         for (Volume vol : VolumeManager.getInstance().getAllVolumes()) {
/* 276 */           switch (vol.getType())
/*     */           {
/*     */           case 1: 
/*     */           case 2: 
/* 280 */             updateMetadata(vol.getId(), new Volume.VolumeMetadata(0, 0, 0));
/*     */           }
/*     */         }
/*     */       }
/*     */       else {
/* 285 */         for (i$ = volumeIds.iterator(); i$.hasNext();) { short volumeId = ((Short)i$.next()).shortValue();
/* 286 */           Volume vol = VolumeManager.getInstance().getVolume(volumeId);
/*     */           
/* 288 */           DbVolumeBlobs.deleteBlobRef(conn, vol);
/*     */           
/* 290 */           updateMetadata(vol.getId(), new Volume.VolumeMetadata(0, 0, 0));
/*     */         }
/*     */       }
/* 293 */       conn.commit();
/*     */     } finally {
/* 295 */       DbPool.quietClose(conn);
/* 296 */       resetProgress();
/*     */     }
/*     */   }
/*     */   
/*     */   private Volume updateMetadata(short volumeId, Volume.VolumeMetadata metadata) throws ServiceException {
/* 301 */     VolumeManager mgr = VolumeManager.getInstance();
/* 302 */     Volume.Builder builder = Volume.builder(mgr.getVolume(volumeId));
/* 303 */     builder.setMetadata(metadata);
/* 304 */     return mgr.update(builder.build());
/*     */   }
/*     */   
/*     */   public void process(List<Short> volumeIds) throws ServiceException, IOException {
/* 308 */     synchronized (this) {
/* 309 */       if (this.inProgress) {
/* 310 */         throw MailServiceException.TRY_AGAIN("Dedupe is already in progress. Only one request can be run at a time.");
/*     */       }
/* 312 */       this.inProgress = true;
/* 313 */       this.totalLinksCreated = 0;
/* 314 */       this.totalSizeSaved = 0L;
/* 315 */       this.volumeBlobsProgress.clear();
/* 316 */       this.blobDigestsProgress.clear();
/*     */     }
/* 318 */     Thread thread = new BlobDeduperThread(volumeIds);
/* 319 */     thread.setName("BlobDeduper");
/* 320 */     thread.start();
/*     */   }
/*     */   
/*     */   private class BlobDeduperThread extends Thread {
/*     */     List<Short> volumeIds;
/*     */     
/*     */     public BlobDeduperThread() {
/* 327 */       this.volumeIds = volumeIds;
/*     */     }
/*     */     
/*     */     private void populateVolumeBlobs(short volumeId, int groupId, int lastSyncDate, int currentSyncDate) throws ServiceException {
/* 331 */       DbPool.DbConnection conn = null;
/* 332 */       Iterable<MailboxBlob.MailboxBlobInfo> allBlobs = null;
/*     */       try {
/* 334 */         conn = DbPool.getConnection();
/* 335 */         allBlobs = DbMailItem.getAllBlobs(conn, groupId, volumeId, lastSyncDate, currentSyncDate);
/* 336 */         for (MailboxBlob.MailboxBlobInfo info : allBlobs) {
/*     */           try {
/* 338 */             DbVolumeBlobs.addBlobReference(conn, info);
/*     */           }
/*     */           catch (MailServiceException se) {
/* 341 */             if (!"mail.ALREADY_EXISTS".equals(se.getCode())) {
/* 342 */               throw se;
/*     */             }
/*     */           }
/*     */         }
/* 346 */         conn.commit();
/*     */       } finally {
/* 348 */         DbPool.quietClose(conn);
/*     */       }
/*     */     }
/*     */     
/*     */     private List<Integer> getSortedGroupIds() throws ServiceException {
/* 353 */       DbPool.DbConnection conn = null;
/*     */       try {
/* 355 */         conn = DbPool.getConnection();
/* 356 */         Set<Integer> groupIds = DbMailbox.getMboxGroupIds(conn);
/* 357 */         List<Integer> groupList = new ArrayList();
/* 358 */         groupList.addAll(groupIds);
/* 359 */         Collections.sort(groupList);
/* 360 */         return groupList;
/*     */       } finally {
/* 362 */         DbPool.quietClose(conn);
/*     */       }
/*     */     }
/*     */     
/*     */     private void populateVolumeBlobs(Volume vol) throws ServiceException {
/* 367 */       Volume.VolumeMetadata metadata = vol.getMetadata();
/* 368 */       boolean resumed = false;
/* 369 */       if (metadata.getCurrentSyncDate() == 0)
/*     */       {
/* 371 */         metadata.setCurrentSyncDate((int)(System.currentTimeMillis() / 1000L));
/*     */       } else {
/* 373 */         resumed = true;
/*     */       }
/* 375 */       if (DebugConfig.disableMailboxGroups) {
/* 376 */         populateVolumeBlobs(vol.getId(), -1, metadata.getLastSyncDate(), metadata.getCurrentSyncDate());
/* 377 */         BlobDeduper.this.setVolumeBlobsProgress(vol.getId(), "1/1");
/*     */       } else {
/* 379 */         List<Integer> groupIds = getSortedGroupIds();
/* 380 */         for (int i = 0; i < groupIds.size(); i++) {
/* 381 */           if ((!resumed) || (((Integer)groupIds.get(i)).intValue() > metadata.getGroupId()))
/*     */           {
/*     */ 
/* 384 */             populateVolumeBlobs(vol.getId(), ((Integer)groupIds.get(i)).intValue(), metadata.getLastSyncDate(), metadata.getCurrentSyncDate());
/* 385 */             metadata.setGroupId(((Integer)groupIds.get(i)).intValue());
/* 386 */             vol = BlobDeduper.this.updateMetadata(vol.getId(), metadata);
/* 387 */             BlobDeduper.this.setVolumeBlobsProgress(vol.getId(), i + 1 + "/" + groupIds.size());
/* 388 */             if (BlobDeduper.this.isStopProcessing()) {
/* 389 */               ZimbraLog.misc.info("Recieved the stop signal. Stopping the deduplication process.");
/* 390 */               throw ServiceException.INTERRUPTED("received stop signal");
/*     */             }
/*     */           }
/*     */         }
/*     */       }
/* 395 */       metadata.setLastSyncDate(metadata.getCurrentSyncDate());
/* 396 */       metadata.setCurrentSyncDate(0);
/* 397 */       metadata.setGroupId(0);
/* 398 */       vol = BlobDeduper.this.updateMetadata(vol.getId(), metadata);
/*     */       
/* 400 */       if (resumed) {
/* 401 */         populateVolumeBlobs(vol);
/*     */       }
/*     */     }
/*     */     
/*     */     public void run() {
/* 406 */       for (Iterator i$ = this.volumeIds.iterator(); i$.hasNext();) { short volumeId = ((Short)i$.next()).shortValue();
/*     */         try {
/* 408 */           ZimbraLog.misc.info("Running deduper for volume %d", new Object[] { Short.valueOf(volumeId) });
/* 409 */           vol = VolumeManager.getInstance().getVolume(volumeId);
/*     */           
/* 411 */           populateVolumeBlobs(vol);
/*     */           
/* 413 */           DbPool.DbConnection conn = null;
/*     */           try {
/* 415 */             conn = DbPool.getConnection();
/* 416 */             digests = DbVolumeBlobs.getUniqueDigests(conn, vol);
/*     */           } finally {
/* 418 */             DbPool.quietClose(conn);
/*     */           }
/* 420 */           count = 0;
/* 421 */           BlobDeduper.this.setBlobDigestsProgress(volumeId, count + "/" + digests.size());
/* 422 */           for (String digest : digests) {
/* 423 */             Pair<Integer, Long> pair = BlobDeduper.this.processDigest(digest, vol);
/* 424 */             BlobDeduper.this.incrementCountAndSize(((Integer)pair.getFirst()).intValue(), ((Long)pair.getSecond()).longValue());
/* 425 */             count++;
/* 426 */             BlobDeduper.this.setBlobDigestsProgress(volumeId, count + "/" + digests.size());
/* 427 */             if (BlobDeduper.this.isStopProcessing()) {
/* 428 */               ZimbraLog.misc.info("Recieved the stop signal. Stopping the deduplication process.");
/* 429 */               break;
/*     */             } } } catch (Throwable t) { Volume vol;
/*     */           SpoolingCache<String> digests;
/*     */           int count;
/* 433 */           ZimbraLog.misc.error("error while performing deduplication", t);
/*     */         } finally {
/* 435 */           BlobDeduper.this.resetProgress();
/*     */         }
/*     */       }
/* 438 */       ZimbraLog.misc.info("Deduping done. Total of " + BlobDeduper.this.totalLinksCreated + " links created and saved approximately " + BlobDeduper.this.totalSizeSaved + " bytes.");
/*     */     }
/*     */   }
/*     */ }


/* Location:              /home/mint/zimbrastore.jar!/com/zimbra/cs/store/file/BlobDeduper.class
 * Java compiler version: 7 (51.0)
 * JD-Core Version:       0.7.1
 */